#!/usr/bin/python
#
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This code is not supported by Google
#

"""
Sample monitoring appengine application for Google Commerce Search instances.

How serving monitors work:

  Every minute this script will issue an empty GCS query (&q=) using the provided 
  cx and API key:
  
  https://www.googleapis.com/shopping/search/v1/cx:<YOUR_CX>/products?country=us&alt=atom&key=<YOUR_KEY>&q=

  If the toalResults for the  query  drops below 70% of the expected number 
  of MerchantCenter items (expected_serving_items)
  or
  if the average response time is greater than 2 seconds
  
  for 3 consecutive mins, this script will send an alert email to <alert_alias>.

  NOTE:  the totalResults number is an estimate and is expected to fluctuate.
  Set expected_serving_items to about 70% of the number of items in your feed for CommerceSearch.
  
  Serving monitoring is just a way to verify GCS service is running and not specifically
  that all your items are returning since totalResults is an estiamte. If there is a system-wide
  serving issue with GCS, then totalResults may fall far below its acceptable variance  (+/- 30% of actual items)
  

How indexing monitors work:
  --> every30 minutes /tasks/mcheartbeat uses the DataFeedsAPI
  (https://developers.google.com/shopping-content/multiclientaccounts/mcdatafeeds)
  to recall the number of items inserted for the CommerceSearch usecase by any feed 
  within the last 30 days.  
  --> For each feed that has the CS usecase set, this script will sum up the number
  items in all those files.
  --> If the number of items in the CS usecase falls below 80% of items
  you expect in to get indexed (expected_indexed_items), this script will send
  an alert email to <alert_alias>.
    
========================================

The default display page is  http://your_app_id.appspot.com 

To use, see:      
http://code.google.com/p/gcs-admin-toolkit/wiki/GCSMonitoringScript

setup:
You need to specify the following variables:
cx:   cx number
api_key:   api key
cid:  merchant center ID
country:  target country for MC and GCS
alert_alias: email alias to send alerts to
expected_serving_items: the number of items expected to get returned as the 'totalResults' count
                        in an empty GCS query (&q=).  Note: the totalResults count is an estimate can
                        can very significantly.  Set expected_serving_items to a number that is about
                        70% of the total number of items
expected_indexed_items the number of items expected to get indexed in the CommerceSearch usecase

Add Google API client library to this appengine project:
APPENGINE_INSTALL_DIR/google_appengine/lib/google-api-python-client/bin/enable-app-engine-project app_engine_id

Add Appengine Identity to Merchant center 
OR
username:  merchant center username
password:  merchant center password
OR
key_file:  Google APIs service account key file in PEM format 
client_id  Google APIs service account name

"""
import os
import webapp2
from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import run_wsgi_app
import urllib, urllib2
from google.appengine.api import users
from google.appengine.ext import db

from datetime import datetime, timedelta
import xml.dom.minidom
import time
import logging
from urllib2 import URLError, HTTPError
from google.appengine.api import mail
from google.appengine.api import app_identity

cx = '<YOUR_CX>'
api_key = '<YOUR_KEY>'
cid = '<YOUR_MERCHANT_ID>'
country = 'us'
alert_alias = ['<YOUR_ALERT_EMAIL@gmail.com']
expected_serving_items = 385000 * 0.7  # expect the totalResults for will return atleast 70% of all your items
expected_indexed_items = 385000

# monitoring using the ContentAPI
# 
# For oauth2 AppIdentity:
# Add the Appengine "Service Account Name" login as an administrator to your
# Merchant Center account.  see: http://code.google.com/p/gcs-admin-toolkit/wiki/GCSMonitoringScript
#
# for ClientLogin:
#username = '<MerchantCenterLogin>'
#password = '<MerchantCenterpassword>'

# for oauth2 Service Accounts:
# first setup an oauth private key: http://code.google.com/p/gcs-admin-toolkit/wiki/GCSAuthentication
# add the oauth2 client_id as an administrator to your merchant center account
# python-crypto does not support der format keys so need to convert the
# default key you get from google into .pem format
# openssl pkcs12 -in your_key.p12 -out your_key.pem -nodes -nocerts -passin pass:notasecret
# edit your_key.pem and delete everything above and below
# -----BEGIN RSA PRIVATE KEY-----
# -----END RSA PRIVATE KEY-----
# note, the default API password for oauth2 is 'notasecret'.  The 
# save the private key in PEM format to the /certs folder in the appengine project
# add the oauth2 'login' name as the client_id below
#key_file = 'certs/<YOUR_OAUTH_KEY>-privatekey.pem'
#client_id = '<YORU_OAUTH_KEY>@developer.gserviceaccount.com'

serving_trip_counter = 0
indexing_trip_counter = 0

class index(webapp2.RequestHandler):
    def get(self):
      user = users.get_current_user()
      logging.info('Access as: ' + user.email())
           
      q = GCSServeLogEntry.all()
      q.order('-date')
      serve_results = q.run(batch_size=4500, limit=4500)  
      
      i = GCSIndexLogEntry.all()
      i.order('-date')
      index_results = i.run(batch_size=100, limit=100)        
               
      template_values = {
            'user': users.get_current_user(),
            'cx' : cx,
            'api_key' : api_key,
            'cid' : cid,
            'country' : country,
            'gd' : serve_results,
            'gi' : index_results,
            'alert_alias' : alert_alias,
            'logout_url': users.create_logout_url(self.request.uri)
      }
        
      path = os.path.join(os.path.dirname(__file__), 'index.html')
      self.response.out.write(template.render(path, template_values))   

class heartbeat(webapp2.RequestHandler):
  
    def get(self):
        user = users.get_current_user()
        try:
            url= ('https://www.googleapis.com/shopping/search/v1/cx:%s/'
                  'products?country=%s&alt=atom&safe=false'
                  '&availability=unknown,outOfStock,limited,inStock,'
                  'backorder,preorder,onDisplayToOrder'
                  '&key=%s') % (cx,country,api_key)

            t1 = time.time()
            xmldoc = self.__executeGCSQuery(url)
            response_time = time.time() - t1

            default_NS = 'http://www.w3.org/2005/Atom'
            openSearch_NS = 'http://a9.com/-/spec/opensearchrss/1.0/'
            s_NS = 'http://www.google.com/shopping/api/schemas/2010'
                   
            n_feed = xmldoc.getElementsByTagNameNS(default_NS, 'feed')[0]
            n_tresults = n_feed.getElementsByTagNameNS(openSearch_NS,
                                                            'totalResults')[0]
            totalResults = int(utilities().getText(n_tresults.childNodes))
            lst_results = []

            nl_product = xmldoc.getElementsByTagNameNS(s_NS,'product')
            for n_product in nl_product:
              ngid = n_product.getElementsByTagNameNS(s_NS,'googleId')
              ntitle = n_product.getElementsByTagNameNS(s_NS,'title')
              npid = n_product.getElementsByTagNameNS(s_NS,'providedId')
              lst_results.append(utilities().getText(npid[0].childNodes))            
              
            logging.info('query ' + url)
            logging.info('response time ' + str(response_time))  
            logging.info('totalResults ' + str(totalResults))
            logging.info('results' + str(lst_results))            

            gd = GCSServeLogEntry(response_time=response_time, totalResults=totalResults)
            #gd.results = lst_results            
            gd.put()
            global serving_trip_counter  
            if (  ( float(float(totalResults)/float(expected_serving_items)) < 0.70) | (float(response_time) > 2) ):
              logging.info('Incrementing serving trip counter ' + str(serving_trip_counter))
              serving_trip_counter = serving_trip_counter + 1
              if (serving_trip_counter == 3):                
                subject=("GCS Alert for " + cx)             
                body = ('GCS System monitor alarm-->  totalResults: ' +
                         str(totalResults) + '  response_time: ' + str(response_time))
                utilities().sendmail(alert_alias, subject, body)
              else:
                if (serving_trip_counter==15):
                  logging.info('Resuming Alert monitor after 15mins')
                  serving_trip_counter = 0                    
            else:
              serving_trip_counter = 0  
                                       
        except Exception, e:
          internal_error = str(e)
          logging.error(internal_error)
          serving_trip_counter += 1
          pass
        
        template_values = {
              'user': user,
              'cx' : cx,
              'api_key' : api_key,
              'cid' : cid,
              'country' : country,
              'message' : 'serving monitor task triggered',
              'alert_alias' : alert_alias,
              'logout_url': users.create_logout_url(self.request.uri)
        }                     
        path = os.path.join(os.path.dirname(__file__), 'index.html')
        self.response.out.write(template.render(path, template_values))       
        
    def __executeGCSQuery(self,url):
      try:
        req = urllib2.Request(url)
        opener = urllib2.urlopen(req)    
        content = opener.read()
        xmldoc = xml.dom.minidom.parseString(content)
        opener.close()
        return xmldoc
      except urllib2.HTTPError, e:
        internal_error = str(e)
        logging.error('HTTP Error ' + internal_error)
      except urllib2.URLError, e:
        internal_error = str(e)
        logging.error('URL Error ' + internal_error)
    
class mcheartbeat(webapp2.RequestHandler):
  
    def get(self):
        user = users.get_current_user()
        auth_token = None
        message = None
        global indexing_trip_counter
        try:         
          url = 'https://content.googleapis.com/content/v1/' + cid + '/datafeeds/products?alt=atom&max-results=25'          
          g = GCSAuthToken()

          #for oauth2 using Service cert-based tokens
          #auth_token = g.getOauth2CertToken(client_id,key_file)
          
          #for oauth2 using Appengine's identity to get Google APIs token
          auth_token = g.getOauth2AppToken()

          if auth_token == None:
            message = 'Unable to acquire oauth2 token'
          else:
            url = url + '&access_token=' + auth_token           
            req = urllib2.Request(url)
            req.add_header('GData-Version', '1')
            req.add_header('Content-type', 'application/atom+xml')
            logging.info("ContentAPI Request for oauth2: " + url)          
                      
          # for ClientLogin
          #auth_token = g.getClientLoginToken(username,password) 
          #if auth_token == None:
          #  message = 'Unable to acquire ClientLogin token'
          #else:
          #  req = urllib2.Request(url)
          #  req.add_header('GData-Version', '1')
          #  req.add_header('Content-type', 'application/atom+xml')
          #  req.add_header('Authorization','GoogleLogin auth=' + auth_token)            
          #  logging.info("ContentAPI Request for ClientLogin: " + url)          
                              
          if (auth_token != None):
            content = urllib2.urlopen(req).read()
            xmldoc = xml.dom.minidom.parseString(content)
                      
            default_NS = 'http://www.w3.org/2005/Atom'
            sc_NS ='http://schemas.google.com/structuredcontent/2009'
            total_items_inserted = 0
            total_items_processed = 0                     
            nl_entries = xmldoc.getElementsByTagNameNS(default_NS, 'entry')
            feed_dict = {}
            feeds_completely_processed = True
            for n_entry in nl_entries:
              title = utilities().getText(n_entry.getElementsByTagNameNS(default_NS,'title')[0].childNodes)
              updated = utilities().getText(n_entry.getElementsByTagNameNS(default_NS,'updated')[0].childNodes)
              asdatetime = datetime.strptime(updated, "%Y-%m-%dT%H:%M:%S.%fZ")
              nl_destinations = n_entry.getElementsByTagNameNS(sc_NS,'feed_destination')
              for n_destination in nl_destinations:
                dest = n_destination.attributes["dest"].value
                enabled = n_destination.attributes["enabled"].value         
                if (dest == 'CommerceSearch' and enabled == 'true'):
                  if (datetime.now() - asdatetime < timedelta(days=30)):
                    logging.info("CommerceSearch enabled feed: " + title + ' ' + str(asdatetime))
                    n_processing_status = n_entry.getElementsByTagNameNS(sc_NS,'processing_status')
                    status = n_processing_status[0].attributes['status'].value
                    if (status == 'processed'):
                      n_items_inserted = n_processing_status[0].getElementsByTagNameNS(sc_NS,'items_inserted')
                      n_items_processed = n_processing_status[0].getElementsByTagNameNS(sc_NS,'items_processed')
                      items_inserted = int(utilities().getText(n_items_inserted[0].childNodes))
                      items_processed = int(utilities().getText(n_items_processed[0].childNodes))
                      logging.info('items_processed/items_inserted :' + str(items_processed) + '/' + str(items_inserted)) 
                      feed_dict[title] = items_inserted                                                                
                      total_items_inserted = total_items_inserted + items_inserted                                
                      total_items_processed =  total_items_processed + items_processed                      
                    elif (status == 'in_progress'):
                      feeds_completely_processed = False
            if (feeds_completely_processed):
              g = GCSIndexLogEntry(items_processed=total_items_processed,items_inserted=total_items_inserted, feed_dict = str(feed_dict))
              g.put()
              if ( float(total_items_inserted)/float(expected_indexed_items) < 0.8):                
                indexing_trip_counter = indexing_trip_counter + 1
                logging.info('Incrementing indexing trip counter ' + str(indexing_trip_counter))                
                if (indexing_trip_counter == 2):                
                  subject = ("GCS Indexing Alert for " + cx + ' merchantID: ' + cid)             
                  body = ('GCS System monitor alarm--> total items_processed: ' +
                           str(items_processed) + '  items_inserted: ' + str(items_inserted))
                  utilities().sendmail(alert_alias, subject, body)
                  indexing_trip_counter = 0
              else:
                indexing_trip_counter = 0        
            else:
              logging.warning('Feed file still processing. Not committing feed monitoring sample.')
              
            message = 'Indexing monitor task triggered'          

        except Exception, e:
          logging.error(str(e))
          message = 'Unable to trigger Indexing monitor'
          pass
        
        template_values = {
              'user': user,
              'cx' : cx,
              'api_key' : api_key,
              'cid' : cid,
              'country' : country,
              'message' : message,              
              'auth_token' : auth_token,
              'alert_alias' : alert_alias,
              'logout_url': users.create_logout_url(self.request.uri)
        }                     
        path = os.path.join(os.path.dirname(__file__), 'index.html')
        self.response.out.write(template.render(path, template_values))      
                          
class GCSAuthToken():
  
    def getOauth2CertToken(self,client_id, key_file):
      #requires pycrypto
      from Crypto.PublicKey import RSA
      from Crypto.Signature import PKCS1_v1_5
      from Crypto.Hash import SHA256
      import json      
      
      #http://code.google.com/p/googleappengine/issues/detail?id=8188
      SCOPES = ['https://www.googleapis.com/auth/shoppingapi',
                'https://www.googleapis.com/auth/structuredcontent']
      SCOPES = ' '.join(SCOPES)

      path = os.path.join(os.path.dirname(__file__), key_file)
      f =file(path, 'rb')
      ikey = RSA.importKey(f.read(),passphrase='notasecret')    
      signer = PKCS1_v1_5.PKCS115_SigScheme(ikey)
      jwt_header = '{"alg":"RS256","typ":"JWT"}'
      
      iss = client_id
      now = int(time.time())
      exptime = now + 3600
      claim =('{"iss":"%s",'
              '"scope":"%s",'
              '"aud":"https://accounts.google.com/o/oauth2/token",'
              '"exp":%s,'
              '"iat":%s}') %(iss,SCOPES,exptime,now)    
  
      jwt = self.__urlsafe_b64encode(jwt_header) + '.' + \
            self.__urlsafe_b64encode(unicode(claim, 'utf-8'))
  
      h = SHA256.new(jwt)
      e = signer.sign(h)        
      assertion = jwt + '.' + self.__urlsafe_b64encode(e)
      url = 'https://accounts.google.com/o/oauth2/token'
      data = {'grant_type' : 'assertion',
              'assertion_type' : 'http://oauth.net/grant_type/jwt/1.0/bearer',
              'assertion' : assertion }
      headers = {"Content-type": "application/x-www-form-urlencoded"}
       
      data = urllib.urlencode(data)
      req = urllib2.Request(url, data, headers)
  
      try:
        resp = urllib2.urlopen(req).read()
        parsed = json.loads(resp)
        access_token = parsed.get('access_token')
        logging.info('Acquired oauth2 access_token: ' + access_token)
        return access_token
      except HTTPError, e:
        logging.error('Error code: ' + str(e.code))
        logging.error(e.read())
        return None
      except URLError, e:
        logging.error( 'Reason: ' + str(e.reason))
        logging.error(e.read())
        return None  
      
    def __urlsafe_b64encode(self,raw_bytes):
      import base64
      return base64.urlsafe_b64encode(raw_bytes).rstrip('=')        

    def getOauth2AppToken(self):
      import httplib2
      from oauth2client.appengine import AppAssertionCredentials
      from apiclient.discovery import build
      
      credentials = AppAssertionCredentials(scope='https://www.googleapis.com/auth/structuredcontent')
      http = credentials.authorize(httplib2.Http())
      # the contentAPI is not available for GoogleAPIs yet so the 'build' wont' work
      # using the 'shopping' build simply to get the 'credentials' object to refresh and
      # actually get a token for structuredcontent (i know, its strange)
      p = build('shopping', 'v1', http=http)     
      return credentials.access_token      
      
    def getClientLoginToken(self, email, passwd):
      #import gdata.client;
      #client = gdata.client.GDClient();
      #client.ClientLogin(email,passwd,"Test","structuredcontent")
      #authToken = client.auth_token.token_string  
           
      authURL = "https://www.google.com/accounts/ClientLogin"
      params = urllib.urlencode({'Email': email, 'Passwd': passwd, 'service':
                                 'structuredcontent', 'source': 'Test'})
      headers = {"Content-type": "application/x-www-form-urlencoded"}
      req = urllib2.Request(authURL, params, headers)
      try:
        response = urllib2.urlopen(req)
      except HTTPError, e:
          logging.error('The server couldnt fulfill the request.')
          logging.error('Error code: ' + str(e.code))
          return None
      except URLError, e:
          logging.error('We failed to reach a server.')
          logging.error( 'Reason: ' + str(e.reason ))
          return None
      lresponse = response.read()
      authResponseDict = dict(x.split("=") for x in lresponse.split("\n") if x)
      if authResponseDict["Auth"] is None:
        logging.info("Unable to acquire token")
        return None
      authToken = authResponseDict["Auth"]
      logging.info("Acquired ClientLogin Token:" + authToken)
      return authToken

class utilities():
  
    def getText(self,nodelist):
      rc = []
      for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
          rc.append(node.data)
      return ''.join(rc)  
    
    def sendmail(self,to, subject, body):
      logging.error('Alert Triggered.  Sending email ' + subject)
      try:             
        app_id = app_identity.get_application_id() 
        sender = 'GCS Monitor <gcsmonitor@' + app_id + '.appspotmail.com>'                                
        message = mail.EmailMessage(sender=sender, subject=subject)               
        message.to = alert_alias
        message.body = (body)              
        message.send()
      except Exception, e:
        internal_error = str(e)
        logging.error(internal_error)
        pass
                  
class GCSServeLogEntry(db.Model):
  user = db.UserProperty()  
  date = db.DateTimeProperty(auto_now_add=True)
  response_time = db.FloatProperty(default=0.000)
  totalResults = db.IntegerProperty(default=0)
  #results = db.ListProperty(str)
  
class GCSIndexLogEntry(db.Model):
  user = db.UserProperty()  
  date = db.DateTimeProperty(auto_now_add=True)
  items_inserted = db.IntegerProperty(default=0)
  items_processed = db.IntegerProperty(default=0)
  feed_dict = db.StringProperty()
  
logging.getLogger().setLevel(logging.INFO) 
application = webapp2.WSGIApplication([('/', index),('/tasks/heartbeat', heartbeat),
                                       ('/tasks/mcheartbeat', mcheartbeat)])

def main():   
  run_wsgi_app(application)

if __name__ == "__main__":
  main()     